{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from lxml.html import fromstring\n",
    "import time,requests,hashlib,base64,json,math,requests_html,os\n",
    "from random import random\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from PIL import Image\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进入知网与文章搜索页"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "# opts.add_argument('--incognito') # 无痕模式\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'D:\\\\爬虫\\\\知网文章-PDF'}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "driver = webdriver.Chrome(options=opts) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(\"https://kns.cnki.net/kns8/defaultresult/index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 检查登录状态\n",
    "element = driver.find_element_by_id('Ecp_loginShowName1')\n",
    "login = element.get_attribute('innerHTML')\n",
    "# 判断并IP登陆\n",
    "if login != '中山大学南方学院':\n",
    "    driver.find_element_by_xpath('//*[@id=\"Ecp_top_login\"]/a').click()\n",
    "    driver.find_element_by_xpath('//*[@id=\"Ecp_Button2\"]').click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "高级检索\n"
     ]
    }
   ],
   "source": [
    "# 进入高级检索\n",
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/a[2]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-29-a8a363765572>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<span>学术期刊</span><em></em>\n"
     ]
    }
   ],
   "source": [
    "# 点击学术期刊\n",
    "element = driver.find_element_by_xpath('//ul[@class=\"doctype-menus keji\"]/li[@data-id=\"xsqk\"]/a')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "专业检索\n"
     ]
    }
   ],
   "source": [
    "# 选择专业检索\n",
    "element = driver.find_element_by_name('majorSearch')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 选择期刊来源\n",
    "\n",
    "## 北大核心来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"HX\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## EI来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"EI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## CSD来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"CSD\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## SCI来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"SI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()\n",
    "## cssci来源期刊\n",
    "element = driver.find_element_by_xpath('//div[@class=\"extend-tit-labels\"]//input[@key = \"CSI\"]')\n",
    "print(element.get_attribute('innerHTML'))\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置搜索query\n",
    "# query = 'SU = \"新媒体\" AND (TI = \"人工智能\" OR TI = \"大数据\" OR TI = \"AI\" OR TI = \"big data\" )'\n",
    "query = \"(SU='媒体融合' AND SU = '大数据') OR  (SU='媒体融合' AND SU = '人工智能')\"\n",
    "element = driver.find_element_by_xpath('//textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//input[@value =\"检索\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导出refworks文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 点击每页显示文章数量\n",
    "time.sleep(5) #等一等\n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/div').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择为50篇\n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/div/i').click()\n",
    "time.sleep(3) \n",
    "driver.find_element_by_xpath('//*[@id=\"perPageDiv\"]/ul/li[3]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#勾选全选\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择“导出与分析”\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]').click()\n",
    "# 选择“导出文件”\n",
    "time.sleep(3) \n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]').click()\n",
    "# 选择Refworks格式\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nfulab\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 导出txt\n",
    "driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 先确定总页数，以便于规定循环翻页的次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 135\n",
      "结果页数总计: 3页\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "from random import random\n",
    "import requests_html\n",
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "结果页数 = html.xpath('//*[@id=\"gridTable\"]/div[2]/span[1]')[0].lxml.text_content()[1:3]\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "\n",
    "pagenum = int(结果页数[0:1])\n",
    "print(pagenum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 清除\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 回到首页\n",
    "driver.find_element_by_xpath('//*[@id=\"total\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-134-704fdf3805c1>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "# 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "已选文献 = driver.find_element_by_xpath('//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "int(已选文献)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "当前页数 = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[2]/span[2]').get_attribute('innerHTML')\n",
    "int(当前页数)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "当前页数 1\n",
      "50\n",
      "当前页数 2\n",
      "100\n",
      "当前页数 3\n",
      "已经全选所有页面文献\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-151-741b839f972a>:33: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "for page in range(1,pagenum+1):\n",
    "#     element = driver.find_element_by_xpath('//html')\n",
    "#     main_content =element.get_attribute('outerHTML')  \n",
    "#     html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "    已选文献 = driver.find_element_by_xpath('//*[@id=\"selectCount\"]').get_attribute('innerHTML')\n",
    "    print(int(已选文献))\n",
    "    当前页数 = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[2]/span[2]').get_attribute('innerHTML')\n",
    "    print(\"当前页数\",int(当前页数))\n",
    "    # Alert Text: 不能超过500个,若要重新选择,请按清除按钮,再进行选取操作\n",
    "    if int(当前页数) < pagenum:\n",
    "        # 勾选本页文献\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/label').click()\n",
    "        # 翻页\n",
    "#         element = driver.find_element_by_xpath('//html')\n",
    "#         main_content =element.get_attribute('outerHTML')  \n",
    "#         main_content\n",
    "        a = driver.find_element_by_xpath('//*[@id=\"PageNext\"]')\n",
    "        a.click()\n",
    "        # 等待时间 平均时间38*103\n",
    "        time.sleep(3+random()*8)\n",
    "    else:\n",
    "        print(\"已经全选所有页面文献\")\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        main_content\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]').click()\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]').click()\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]').click()\n",
    "# ----------------------------进入导出页面------------------------------------\n",
    "        driver.switch_to_window(driver.window_handles[2])\n",
    "        time.sleep(3+random()*8)\n",
    "        driver.find_element_by_xpath('/html/body/div[3]/div/div[1]/ul/li[8]').click()\n",
    "        driver.find_element_by_xpath('//*[@id=\"litotxt\"]').click()\n",
    "# ----------------------------清除已选文章，继续翻页------------------------------------\n",
    "        driver.switch_to.window(driver.window_handles[1])\n",
    "        element = driver.find_element_by_xpath('//html')\n",
    "        main_content =element.get_attribute('outerHTML') \n",
    "        a = driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a').click()\n",
    "        time.sleep(3+random()*10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文章下载——PDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 点击谋篇文章标题\n",
    "driver.find_element_by_xpath('//*[@id=\"gridTable\"]/table/tbody/tr[8]/td[2]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-50-a8a363765572>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "## 切换窗口\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 点击pdf下载\n",
    "driver.find_element_by_xpath('//*[@id=\"pdfDown\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有验证码，则弹出新页面--->换窗口\n",
    "driver.switch_to_window(driver.window_handles[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 对验证码进行截图\n",
    "# encoding:utf-8\n",
    "from PIL import Image\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    " \n",
    "# url = 'https://weixin.sogou.com/antispider/?from=http%3A%2F%2Fweixin.sogou.com%2Fweixin%3Ftype%3D2%26query%3Dpython'\n",
    "# driver = webdriver.Chrome()\n",
    "# driver.maximize_window()  # 将浏览器最大化\n",
    "# driver.get(url)\n",
    "# # 截取当前网页并放到D盘下命名为printscreen，该网页有我们需要的验证码\n",
    "driver.save_screenshot('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\web_imgprintscreen.png')\n",
    "imgelement = driver.find_element_by_id('vImg')  # 定位验证码\n",
    "location = imgelement.location  # 获取验证码x,y轴坐标\n",
    "print(location)\n",
    "size = imgelement.size  # 获取验证码的长宽\n",
    "print(size)\n",
    "rangle = (int(location['x']+260), int(location['y']+135), int(location['x'] + size['width']+360),\n",
    "          int(location['y'] + size['height']+170))  # 写成我们需要截取的位置坐标\n",
    "i = Image.open(\"C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\web_imgprintscreen.png\")  # 打开截图\n",
    "frame4 = i.crop(rangle)  # 使用Image的crop函数，从截图中再次截取我们需要的区域\n",
    "frame4=frame4.convert('RGB')\n",
    "frame4.save('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg') # 保存我们接下来的验证码图片 进行打码\n",
    " \n",
    "# driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 讯飞OCR识别验证码\n",
    "\"\"\"\n",
    "  印刷文字识别WebAPI接口调用示例接口文档(必看)：https://doc.xfyun.cn/rest_api/%E5%8D%B0%E5%88%B7%E6%96%87%E5%AD%97%E8%AF%86%E5%88%AB.html\n",
    "  上传图片base64编码后进行urlencode要求base64编码和urlencode后大小不超过4M最短边至少15px，最长边最大4096px支持jpg/png/bmp格式\n",
    "  (Very Important)创建完webapi应用添加合成服务之后一定要设置ip白名单，找到控制台--我的应用--设置ip白名单，如何设置参考：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=41891\n",
    "  错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "  @author iflytek\n",
    "\"\"\"\n",
    "#-*- coding: utf-8 -*-\n",
    "import requests\n",
    "import time\n",
    "import hashlib\n",
    "import base64\n",
    "import json\n",
    "#from urllib import parse\n",
    "# 印刷文字识别 webapi 接口地址\n",
    "URL = \"http://webapi.xfyun.cn/v1/service/v1/ocr/general\"\n",
    "# 应用ID (必须为webapi类型应用，并印刷文字识别服务，参考帖子如何创建一个webapi应用：http://bbs.xfyun.cn/forum.php?mod=viewthread&tid=36481)\n",
    "APPID = \"600a7fb3\"\n",
    "# 接口密钥(webapi类型应用开通印刷文字识别服务后，控制台--我的应用---印刷文字识别---服务的apikey)\n",
    "API_KEY = \"73071c3d054a8bfc81f2ef13debafb91\"\n",
    "def getHeader():\n",
    "#  当前时间戳\n",
    "    curTime = str(int(time.time()))\n",
    "#  支持语言类型和是否开启位置定位(默认否)\n",
    "    param = {\"language\": \"cn|en\", \"location\": \"false\"}\n",
    "    param = json.dumps(param)\n",
    "    paramBase64 = base64.b64encode(param.encode('utf-8'))\n",
    "\n",
    "    m2 = hashlib.md5()\n",
    "    str1 = API_KEY + curTime + str(paramBase64,'utf-8')\n",
    "    m2.update(str1.encode('utf-8'))\n",
    "    checkSum = m2.hexdigest()\n",
    "# 组装http请求头\n",
    "    header = {\n",
    "        'X-CurTime': curTime,\n",
    "        'X-Param': paramBase64,\n",
    "        'X-Appid': APPID,\n",
    "        'X-CheckSum': checkSum,\n",
    "        'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',\n",
    "    }\n",
    "    return header\n",
    "#上传文件并进行base64位编码\n",
    "with open(r'C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg', 'rb') as f:\n",
    "    f1 = f.read()\n",
    "\n",
    "f1_base64 = str(base64.b64encode(f1), 'utf-8')\n",
    "\n",
    "# ## 网络外链转换base64位编码\n",
    "\n",
    "# f2_base64 = str(base64.b64encode(base64data), 'utf-8')\n",
    "\n",
    "\n",
    "data = {\n",
    "        'image':f1_base64\n",
    "        }\n",
    "\n",
    "\n",
    "\n",
    "r = requests.post(URL, data=data, headers=getHeader())\n",
    "result = str(r.content, 'utf-8')\n",
    "# # 错误码链接：https://www.xfyun.cn/document/error-code (code返回错误码时必看)\n",
    "# print(result)\n",
    "# input(\"Entry the any key to exit\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 观察返回值\n",
    "r.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 若有OCR返回值，得到验证码\n",
    "yzm = r.json()['data']['block'][0]['line'][0]['word'][0]['content'()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输入验证码\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "## 提交\n",
    "time.sleep(3)\n",
    "driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动爬取"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第一步：获取全部URL队列 \n",
    "* 尽量减少循环步骤\n",
    "* 尽量减少打开网页的次数（网络不稳定等不可抗力）\n",
    "* 便于爬取大量文章时出现中断，继续爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_link():\n",
    "    html=driver.page_source\n",
    "    root=fromstring(html)\n",
    "    href = [x for x in root.xpath('.//*[@id=\"gridTable\"]/table/tbody/tr/td[2]/a/@href')]\n",
    "    c_link=['https://kns.cnki.net/kcms/detail/detail.aspx?'+x.split('&')[6]+'&'+x.split('&')[5]+'&'+x.split('&')[4] for x in href]\n",
    "    return c_link\n",
    "    # del c_link[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "down_xpath = '//*[@id=\"gridTable\"]/table/tbody/tr['+str(i)+']/td[8]'\n",
    "d = driver.find_element_by_xpath(down_xpath).get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "836"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "文章数量总计: 381\n",
      "结果页数总计: 20\n"
     ]
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "文章数量 = html.xpath('//*[@id=\"countPageDiv\"]/span[1]/em')[0].lxml.text_content()\n",
    "print(\"文章数量总计:\",文章数量)\n",
    "结果页数 = html.xpath('//*[@id=\"gridTable\"]/div[2]/span[1]')[0].lxml.text_content()[1:3]\n",
    "print(\"结果页数总计:\",结果页数)\n",
    "pagenum = int(结果页数)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pagenum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-69-87d9b9498e06>:13: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1]) #切换窗口\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "翻页完成\n",
      "_________________________________\n"
     ]
    }
   ],
   "source": [
    "all_link = []\n",
    "use_index = []\n",
    "\n",
    "\n",
    "for page in range(1,pagenum+1):\n",
    "    print(page)\n",
    "    time.sleep(3)\n",
    "    k = get_link()\n",
    "    lock_index = []\n",
    "#     print(len(k))\n",
    "    for i in range(1,21): ##50篇一页or20篇一页\n",
    "        \n",
    "        driver.switch_to_window(driver.window_handles[1]) #切换窗口\n",
    "        down_xpath = '//*[@id=\"gridTable\"]/table/tbody/tr['+str(i)+']/td[8]'\n",
    "        try:\n",
    "            d = driver.find_element_by_xpath(down_xpath).get_attribute('innerHTML')\n",
    "        except:\n",
    "            print(\"翻页完成\")\n",
    "        if len(d)>700:  ##说明文章未被锁，正常  \n",
    "            a1 = i-1\n",
    "#             use_index.append[a1]\n",
    "        else:\n",
    "            a2 = i-1\n",
    "            lock_index.append(a2)\n",
    "            print(page,lock_index)\n",
    "            for j in lock_index:\n",
    "                del k[j-1]\n",
    "    all_link.extend(k)\n",
    "    page += 1   \n",
    "    try:\n",
    "        driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "    except:\n",
    "        print(\"_________________________________\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"total\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "381"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\"文献详细页链接\":all_link}\n",
    "data = pd.DataFrame(data)\n",
    "data.to_excel(\"链接.xlsx\",sheet_name=\"sheetname\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第二步：下载PDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = pd.read_excel('链接.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202111014&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202110047&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB202103007&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202110012&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB202103007&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202110012&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX202105009&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX202105009&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202109018&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202109018&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY202105005&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202108022&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202108022&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202107034&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202107034&dbname=CJFDAUTO',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202107013&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202107002&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202104003&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202104008&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202106011&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202105013&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202105031&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202105014&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDDS202103029&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202104021&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202104001&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202104003&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK202102032&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZZDX202101020&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB202101005&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202101005&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202101005&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB2020S1054&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202036028&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202023020&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202023020&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202034023&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202012005&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202012017&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202022022&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202033026&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=42.1618.G2.20201126.2010.010&filename=CBKX202006006&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=HNDB202006012&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=LLSY202011014&dbname=CJFDLAST2021',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202021014&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202021004&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202021006&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202011011&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDDS202009021&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202024011&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=HNDS202005002&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=11.3209.G3.20200817.1648.005&filename=KJYU202008010&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202015013&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202015013&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202008009&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202021028&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202007003&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XXDJ202013008&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK202007035&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202012006&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202017030&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202011004&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202016022&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB202011011&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=RMLT202016028&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI202006016&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI202006018&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=11.3209.G3.20200601.1335.026&filename=KJYU202006010&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202014022&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202014050&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB202003003&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGDD202005012&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=SHDS202003011&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG202005014&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202005018&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202012008&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJXB202002022&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB202004006&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202006026&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ202005016&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY202003003&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB202002008&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202003005&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202004009&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY202002014&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY202002013&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB202001004&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202001005&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202001027&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202001005&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI202001007&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202001007&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ202001008&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJXB2019S2050&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XZGL201912005&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201923009&dbname=CJFDLAST2020',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20191125.1524.013&filename=CBGJ201920015&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20191122.1609.017&filename=CBGJ201921019&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201922005&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201922008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JYKQ201911008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JYKQ201911004&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201911014&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201930023&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201920006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201919017&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX201909027&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201917036&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=11.3209.G3.20190903.1638.002&filename=KJYU201909015&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201909005&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201909039&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWAH201908004&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201908003&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX201908004&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201915010&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG201908007&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201908004&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201908016&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201908028&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201908029&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDDS201908026&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZXGL201908015&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201921014&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201914027&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GXSJ201907028&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=NJSH201907017&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JYKQ201907017&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201918009&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG201906004&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201906034&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201906047&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201906013&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201915008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201913032&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201913034&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201913011&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB201903006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=KJYU201905001&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201905005&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201905006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201905008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201905021&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201905006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201905009&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201905018&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=RMXS201907012&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201908009&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX201904010&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201907016&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201910014&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20190326.1603.006&filename=CBGJ201903008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201906025&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JYKQ201903003&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201907012&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DXCB201902011&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201903018&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201903008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201906003&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201904006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201904002&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201902002&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY201902008&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201902005&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201901007&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201901005&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=RMXS201824001&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=RMXS201824006&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201836011&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=NEWS201811013&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201811014&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201830030&dbname=CJFDLAST2019',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=YJSX201805023&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201819001&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201819009&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=FLSH201805029&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201818009&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBFX201809022&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201805023&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201817033&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201817004&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201817005&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201816005&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201823014&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201808008&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201807002&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201807004&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201807008&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201812011&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201817066&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG201806018&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201806044&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20180604.0607.025&filename=CBGJ201810027&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201810012&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201809008&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201809019&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201813019&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201805019&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201808004&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201808023&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201804005&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201804009&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20180403.1123.006&filename=CBGJ201805008&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201806031&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20180322.1428.002&filename=CBGJ201804002&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201803011&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201803016&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201803010&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWJZ201803008&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201803006&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20180208.1048.004&filename=CBGJ201803004&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20180208.1048.002&filename=CBGJ201803002&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201802018&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201801025&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201802019&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201802027&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201801004&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201801012&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201801013&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201801038&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201801035&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201801020&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201801022&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201712027&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201712041&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201712045&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=NEWS201712012&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JYKQ201712001&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201712033&dbname=CJFDLAST2018',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZZXK201711029&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201720015&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201728027&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=KJYU201710019&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201710004&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QUIT201709021&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201718007&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201725004&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201717011&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201717017&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201717026&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201709012&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=KJTB201708059&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201716005&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201708027&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201715013&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201715017&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201714011&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201714012&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201714001&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201714003&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201713015&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201713001&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201707007&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201707025&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201706003&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201711051&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201711023&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201706034&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201706007&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201706012&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201710004&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XNZS201705028&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201709050&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XXYS201704015&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201707017&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201704018&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201704034&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201704007&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201708017&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201702026&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201707012&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201707008&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201703010&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201703002&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201702018&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201702015&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201701002&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201701006&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201701012&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201701013&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GSSH201606018&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201621022&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201611017&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201611002&dbname=CJFDLAST2017',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.g2.20161020.1403.014&filename=CBGJ201617007&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWAH201610010&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201619009&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201610003&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201627003&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201617030&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201617050&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201609005&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201609006&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201609008&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201609045&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201624004&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201608029&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201615004&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201615033&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201608008&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZSDB201604016&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201613007&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JHLZ201604024&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201607004&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201612005&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGTJ201606010&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201606005&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201614032&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201605008&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201605010&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201605008&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWDX201602017&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201607006&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201606003&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201606066&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201603015&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201606055&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=SHKJ201602033&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=SXDA201601025&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGCB201602002&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201601032&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201601030&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201601017&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JHKX201506038&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201512009&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=SXDA201506010&dbname=CJFDLAST2016',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=JJTL201511011&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201531048&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=KJYU201511023&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201521029&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201521002&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201511004&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DLXS201506030&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201511024&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201529057&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201519011&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201519036&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201518034&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGDD201509022&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=11.1322.J.20150914.1714.072&filename=DDDS201509036&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=NEWS201517008&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZY201509021&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201509034&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201509065&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XDCB201508005&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=45.1216.G2.20150811.1646.104&filename=CBGJ201506009&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=KFDB201504026&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201508005&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201508014&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201508028&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201514035&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201513026&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201516007&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201511006&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201514057&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201509013&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CBGJ201505037&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWAH201504017&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CMEI201506031&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201506027&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DACB201502002&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=BJZG201502030&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201507045&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWJZ201503004&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDDS201503005&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201503009&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GDXK201503014&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201505043&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=GJXW201501003&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=CQSK201501013&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201501016&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=ZGJZ201501065&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201434033&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201422009&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWXZ201411015&dbname=CJFDLAST2015',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=XWZX201410016&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201418009&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201424021&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ201417052&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DDJY201403014&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSSS201402030&dbname=CJFD2014',\n",
       " 'https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=DSYI201401023&dbname=CJFD2014']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_list = url[\"文献详细页链接\"].tolist()\n",
    "url_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 封装函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def down_PDF():\n",
    "    driver.find_element_by_xpath('//*[@id=\"pdfDown\"]').click()\n",
    "    time.sleep(30)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def switch_win():\n",
    "    time.sleep(5)\n",
    "    now_page = len(driver.window_handles)\n",
    "    print(\"当前页面数\",now_page)\n",
    "    driver.switch_to_window(driver.window_handles[now_page-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_isTrue():\n",
    "    flag=True\n",
    "    try:\n",
    "        driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "        return flag\n",
    "    except:\n",
    "        flag=False\n",
    "        return flag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 尝试用图鉴\n",
    "\n",
    "import base64\n",
    "import json\n",
    "import requests\n",
    "\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def OCR():\n",
    "    time.sleep(5)\n",
    "    # 截图\n",
    "    element=driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "    # 选择存储路径\n",
    "    element.screenshot('C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg')\n",
    "    # api\n",
    "    yzm = base64_api(uname='trevor', pwd='18078846977', img=\"C:\\\\Users\\\\zjl\\\\Desktop\\\\web数据挖掘\\\\web_img\\\\save.jpg\", typeid=3)\n",
    "    isalnum_yzm = yzm.isalnum()\n",
    "    if len(yzm) == 4 and isalnum_yzm == True:\n",
    "        # 填写/提交验证码\n",
    "        time.sleep(5)\n",
    "        driver.find_element_by_xpath('//*[@id=\"vcode\"]').send_keys(yzm)\n",
    "        print(\"可提交的验证码：\",yzm)\n",
    "        time.sleep(5)\n",
    "        driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click()\n",
    "#             pop_window_isTrue()\n",
    "        results = True\n",
    "        time.sleep(30)\n",
    "    else:\n",
    "        # 点击验证码图片\n",
    "        time.sleep(5)\n",
    "        print(\"换验证码\")\n",
    "        driver.find_element_by_xpath('//*[@id=\"vImg\"]').click() ##换验证码\n",
    "        isimgTrue = True\n",
    "        results = False\n",
    "\n",
    "    return results     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_file():\n",
    "    E = os.listdir(\"D:\\\\爬虫\\\\测试\")\n",
    "    file_nmb = len(E)\n",
    "    return file_nmb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## 加载时间过长\n",
    "def timeout():\n",
    "    print(\"进行超时处理\")\n",
    "    driver.set_page_load_timeout(10)\n",
    "    while True:\n",
    "        try:\n",
    "            new_url = driver.current_url\n",
    "            driver.get(new_url)\n",
    "            break\n",
    "        except:\n",
    "            pass  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 判断是否出现弹窗\n",
    "## 代码优化后 这个函数可以不用到\n",
    "def pop_window_isTrue():\n",
    "    try:\n",
    "        driver.switch_to.alert\n",
    "        print(\"出现弹窗\")\n",
    "        return True\n",
    "        \n",
    "    except:\n",
    "        print(\"未出现弹窗\")\n",
    "        return False\n",
    "        pass\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "未出现弹窗\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pop_window_isTrue()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自动下载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "# opts.add_argument('--incognito') # 无痕模式\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'D:\\\\爬虫\\\\测试'}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "# driver = webdriver.Chrome(options=opts) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts.add_argument(\"start-maximized\")\n",
    "opts.add_argument(\"enable-automation\")\n",
    "opts.add_argument(\"--disable-infobars\")\n",
    "opts.add_argument(\"--disable-dev-shm-usage\")\n",
    "opts.add_argument(\"--disable-browser-side-navigation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 40\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=&filename=QNJZ202033026&dbname=CJFDLAST2020\n",
      "进行超时处理\n",
      "hov_file_nmb: 40\n",
      "进行超时处理\n",
      "file_nmb: 40\n",
      "isDownLoadTrue： True\n",
      "文章数量未增加\n",
      "当前页面数 2\n",
      "isimgTrue： True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-6133cfe30bad>:5: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[now_page-1])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当前页面数 2\n",
      "验证码前hov_file_nmb: 40\n",
      "可提交的验证码： rga2\n",
      "验证码后file_nmb: 41\n",
      "下载成功，文章数量增加\n",
      "关闭页面，2.i+1\n",
      "当前页面数 1\n",
      "关闭页面\n",
      "【i=】 41\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 41\n",
      "进行超时处理\n",
      "https://kns.cnki.net/kcms/detail/detail.aspx?urlid=42.1618.G2.20201126.2010.010&filename=CBKX202006006&dbname=CJFDLAST2020\n",
      "进行超时处理\n",
      "hov_file_nmb: 41\n",
      "进行超时处理\n",
      "file_nmb: 41\n",
      "isDownLoadTrue： True\n",
      "文章数量未增加\n",
      "当前页面数 2\n",
      "isimgTrue： True\n",
      "当前页面数 2\n",
      "验证码前hov_file_nmb: 41\n",
      "可提交的验证码： s5f2\n",
      "验证码后file_nmb: 42\n",
      "下载成功，文章数量增加\n",
      "关闭页面，2.i+1\n",
      "当前页面数 1\n",
      "关闭页面\n",
      "【i=】 42\n",
      "------------------开始下一篇文章--------------------\n",
      "此篇文章：i= 42\n"
     ]
    },
    {
     "ename": "TimeoutException",
     "evalue": "Message: timeout: Timed out receiving message from renderer: 300.000\n  (Session info: chrome=91.0.4472.101)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTimeoutException\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-26-bdccc4f7243f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mdriver\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwebdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mChrome\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mopts\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mdriver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl_list\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[0mtimeout\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl_list\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, url)\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[0mLoads\u001b[0m \u001b[0ma\u001b[0m \u001b[0mweb\u001b[0m \u001b[0mpage\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mcurrent\u001b[0m \u001b[0mbrowser\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    332\u001b[0m         \"\"\"\n\u001b[1;32m--> 333\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGET\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'url'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0murl\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    334\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    335\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[1;34m(self, driver_command, params)\u001b[0m\n\u001b[0;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 321\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[0;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[1;34m(self, response)\u001b[0m\n\u001b[0;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'alert'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    241\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 242\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    243\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    244\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTimeoutException\u001b[0m: Message: timeout: Timed out receiving message from renderer: 300.000\n  (Session info: chrome=91.0.4472.101)\n"
     ]
    }
   ],
   "source": [
    "isDownLoadTrue = True\n",
    "isimgTrue = True\n",
    "\n",
    "for i in range(40,len(url_list)):\n",
    "    print(\"------------------开始下一篇文章--------------------\")\n",
    "    print(\"此篇文章：i=\",i)\n",
    "    isDownLoadTrue = True\n",
    "    driver = webdriver.Chrome(options=opts) \n",
    "    time.sleep(3)\n",
    "    driver.get(url_list[i])\n",
    "    timeout()\n",
    "    print(url_list[i])\n",
    "    time.sleep(5)\n",
    "    timeout()\n",
    "    hov_file_nmb = check_file()\n",
    "    print(\"hov_file_nmb:\",hov_file_nmb)\n",
    "    down_PDF()\n",
    "    timeout()\n",
    "    file_nmb = check_file()\n",
    "    print(\"file_nmb:\",file_nmb)\n",
    "    print(\"isDownLoadTrue：\",isDownLoadTrue)\n",
    "    while isDownLoadTrue:        \n",
    "        if file_nmb>hov_file_nmb:\n",
    "            print(\"下载成功，文章数量增加\")\n",
    "            i += 1\n",
    "            isDownLoadTrue = False\n",
    "            print(\"关闭页面，1.i+1\")\n",
    "            driver.close()#\"关掉页面\"   \n",
    "        else:\n",
    "            print(\"文章数量未增加\")\n",
    "            switch_win()\n",
    "            print(\"isimgTrue：\",isimgTrue)\n",
    "            while isimgTrue:\n",
    "                switch_win()\n",
    "                if img_isTrue():\n",
    "                    hov_file_nmb = check_file()\n",
    "                    print(\"验证码前hov_file_nmb:\",hov_file_nmb)\n",
    "                    results = OCR()\n",
    "                    file_nmb = check_file()\n",
    "                    print(\"验证码后file_nmb:\",file_nmb)\n",
    "                    if results == True:\n",
    "                            if img_isTrue()==False:\n",
    "                                print(\"并发or验证码错误过多or出现弹窗——关闭当前页面-切换窗口-重新点击下载PDF\")\n",
    "                                driver.close()#\"关掉页面\" \n",
    "                                i = i\n",
    "                                print(\"{i=:}\",i)            \n",
    "                            else:\n",
    "                                if file_nmb>hov_file_nmb:\n",
    "                                    print(\"下载成功，文章数量增加\")\n",
    "                                    i += 1\n",
    "                                    isDownLoadTrue = False\n",
    "                                    print(\"关闭页面，2.i+1\")\n",
    "                                    driver.close()#\"关掉页面\"    \n",
    "                                else:\n",
    "                                    print(\"换验证码\")\n",
    "                                    driver.find_element_by_xpath('//*[@id=\"vImg\"]').click() ##换验证码\n",
    "                                    print(\"文章数量未增加\") \n",
    "                                    isimgTrue = True\n",
    "                        \n",
    "                    else:\n",
    "                        #验证码错误or出现弹窗——关闭当前页面-切换窗口-重新点击下载PDF\n",
    "                        isimgTrue = True                        \n",
    "                else:\n",
    "                    print(\"关闭页面\")\n",
    "                    driver.close()#\"关掉页面\"                    \n",
    "                    print(\"【i=】\",i)\n",
    "                    break\n",
    "\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "element = driver.find_element_by_xpath('//html')\n",
    "main_content =element.get_attribute('outerHTML')  \n",
    "html = requests_html.HTML(html= main_content, url='https://localhost/')\n",
    "data_list = html.xpath('//*[@id=\"vImg\"]')\n",
    "data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "164.988px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
